package com.jandar.spider.task;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.jandar.spider.config.SeleniumConfig;
import com.jandar.spider.entity.ContentBiddingEntity;
import com.jandar.spider.mapper.ContentBiddingMapper;
import com.jandar.spider.utils.OkHttpUtils;
import com.jandar.spider.utils.SeleniumUtil;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import okhttp3.OkHttpClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.WebDriver;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

@Slf4j
@Component
public class ContentBiddingTask {
    @Autowired
    private SeleniumConfig seleniumConfig;
    @Autowired
    private ContentBiddingMapper mapper;
    @Autowired
    private OkHttpUtils okHttpUtils;
    private final static BlockingQueue<ContentBiddingEntity> dataQueue = new LinkedBlockingDeque<>();
    private final static ConcurrentHashMap<String, TaskStatus> taskStatusMap = new ConcurrentHashMap<>();
    private final int canRepeat = 100;

    @Async("taskExecutor")
    public void producer(String name) {
        taskStatusMap.putIfAbsent(name, new TaskStatus());
        taskStatusMap.get(name).getRun().set(true);
        taskStatusMap.get(name).getRepeatCount().set(0);
        try {

            switch (name) {
                case "ZJ":
                    ZJProduce();
                    break;
                case "HB":
                    HBProduce();
                    break;
                case "BJ":
                    BJProduce();
                    break;
                case "JS":
                    JSProduce();
                    break;
                default:
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            taskStatusMap.remove(name);
            log.info("produce over");
        }
    }

    @Async("taskExecutor")
    public void consumer() {
        do {
            if (dataQueue.isEmpty()) {
                try {
                    log.info("让队列拿数据....，休息1分钟");
                    Thread.sleep(60000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } else {
                ContentBiddingEntity contentBiddingEntity = dataQueue.poll();
                try {
                    mapper.insert(contentBiddingEntity);
                    log.info("插入成功:{}", contentBiddingEntity.getCode());
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        } while (!taskStatusMap.isEmpty());
        log.info("consume over");
    }


    protected void HBProduce() {
        final AtomicInteger pageNumber = new AtomicInteger(0);
        do {
            String url = "http://search.hebcz.cn:8080/was5/web/search?page=" + pageNumber.incrementAndGet() + "&channelid=240117&perpage=50&outlinepage=10&lanmu=zbgg";
            log.info("当前访问路径：" + url);
            Element body = null;
            try {
                body = Jsoup.connect(url).timeout(30000).get();
            } catch (Exception e) {
                e.printStackTrace();
                continue;
            }
            List<Element> titleList = body.select("#biaoti");
            if (titleList.isEmpty() || taskStatusMap.get("HB").getRepeatCount().get() > canRepeat) {
                taskStatusMap.get("HB").getRun().set(false);
            }
            List<Element> infoList = body.select("#biaoti + tr");
            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
            for (int i = 0; i < infoList.size() && i < titleList.size(); i++) {
                log.info("当前队列数量：" + dataQueue.size());
                ContentBiddingEntity contentBiddingEntity = new ContentBiddingEntity();
                Element a = titleList.get(i).select("td > a").get(0);
                String uri = a.attr("href");
                String title = a.text();
                List<Element> info = infoList.get(i).select("td > span");
                String pubDate = info.get(0).text();
                String area = info.get(1).text();
                String buyer = info.get(2).text();

                try {
                    contentBiddingEntity.setPubDate(format.parse(pubDate).getTime());
                } catch (ParseException e) {
                    e.printStackTrace();
                }
                contentBiddingEntity.setDistrictName(area);
                contentBiddingEntity.setTitle(title);
                contentBiddingEntity.setProvince("河北政府采购网");
                Document contentHtml = null;
                try {
                    contentHtml = Jsoup.connect(uri).post();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                try {
                    List<Element> codeSelector = contentHtml.select("#2020_VERSION > tbody > tr:nth-child(10) > td > span > span:nth-child(2)");
                    List<Element> projectNameSelector = contentHtml.select("#2020_VERSION > tbody > tr:nth-child(10) > td > span > span:nth-child(4)");
                    if (codeSelector.isEmpty()) {
                        codeSelector = contentHtml.select("body > table > tbody > tr > td > table > tbody > tr:nth-child(4) > td > table > tbody > tr:nth-child(1) > td > table > tbody > tr:nth-child(1) > td:nth-child(2)");
                    }
                    if (projectNameSelector.isEmpty()) {
                        projectNameSelector = contentHtml.select("body > table > tbody > tr > td > table > tbody > tr:nth-child(4) > td > table > tbody > tr:nth-child(1) > td > table > tbody > tr:nth-child(2) > td.txt1");
                    }
                    String code = codeSelector.get(0).text();
                    String projectName = projectNameSelector.get(0).text();
                    contentBiddingEntity.setContentHtml(contentHtml.html());
                    contentBiddingEntity.setCode(code);
                    contentBiddingEntity.setTypeName("公开招标");
                    contentBiddingEntity.setProjectName(projectName);
                    contentBiddingEntity.setUrl(uri);
                    Thread.sleep(2000);
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error(uri);
                }
                if (isNotExist(contentBiddingEntity.getCode(), contentBiddingEntity.getPubDate(), "HB")) {
                    dataQueue.add(contentBiddingEntity);
                }
            }
        } while (taskStatusMap.get("HB").getRun().get());
    }


    private synchronized JSONArray postZJ(JSONObject param) {
        sleep();
        try {
            String api = "https://zfcg.czt.zj.gov.cn/portal/category";
            log.info("post zj:" + api);
            JSONArray resp = okHttpUtils.okhttpPost(api, param).getJSONObject("result").getJSONObject("data").getJSONArray("data");
            if (resp != null) {
                return resp;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new JSONArray();

    }

    public static void main(String[] args) {
        OkHttpClient client = new OkHttpClient.Builder()
                .proxy(new java.net.Proxy(java.net.Proxy.Type.HTTP, new java.net.InetSocketAddress("183.221.242.104", 9443)
                )).build();
        String url = "http://myip.ipip.net";
        okhttp3.Request request = new okhttp3.Request.Builder().url(url).build();
        try {
            okhttp3.Response response = client.newCall(request).execute();
            System.out.println(response.body().string());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    // 招标公告
    protected void ZJProduce() {
        final AtomicInteger pageNumber = new AtomicInteger(0);
        do {

            JSONObject param = new JSONObject();
            param.put("pageNo", pageNumber.incrementAndGet());
            param.put("pageSize", "100");
            param.put("isGov", true);
            JSONArray excludeDistrictPrefix = new JSONArray();
            excludeDistrictPrefix.add("90");
            excludeDistrictPrefix.add("006011");
            param.put("excludeDistrictPrefix", excludeDistrictPrefix);
            JSONArray array = new JSONArray();
            // 招标公告
            param.put("categoryCode", "110-684034");
            array.addAll(postZJ(param));
            // 中标结果公告
            param.put("categoryCode", "110-188043");
            array.addAll(postZJ(param));
            // 采购意向公告
//            param.put("categoryCode", "110-600268");
//            array.addAll(postZJ(param));

            if (array.size() == 0 || taskStatusMap.get("ZJ").getRepeatCount().get() > canRepeat) {
                taskStatusMap.get("ZJ").getRun().set(false);
                return;
            }
            for (int i = 0; i < array.size(); i++) {

                JSONObject data = array.getJSONObject(i);
                ContentBiddingEntity contentBiddingEntity = new ContentBiddingEntity();
                contentBiddingEntity.setCode(data.getString("articleId"));
                contentBiddingEntity.setMainBidMenuName(data.getString("gpCatalogName"));
                contentBiddingEntity.setTitle(data.getString("title"));
                contentBiddingEntity.setProvince("浙江省政府采购网");
                contentBiddingEntity.setProjectCode(data.getString("projectCode"));
                contentBiddingEntity.setProjectName(data.getString("projectName"));
                contentBiddingEntity.setPubDate(data.getLong("publishDate"));
                contentBiddingEntity.setDistrictName(data.getString("districtName"));
//                contentBiddingEntity.setType(data.getInteger("type"));
                contentBiddingEntity.setTypeName(data.getString("procurementMethod"));
                contentBiddingEntity.setKeywords(data.getString("keywords"));
                contentBiddingEntity.setRemark(data.getString("remark"));
                String url = "https://zfcg.czt.zj.gov.cn/luban/detail?articleId=" + data.getString("articleId").replace("+","%2B").replace("=","%3D");
                contentBiddingEntity.setUrl(url);
                try {
                    Thread.sleep(30000);
                    JSONObject noticeDetail = getNoticeDetail(data.getString("articleId").replace("+", "%2B").replace("=", "%3D"));
                    contentBiddingEntity.setContentHtml(noticeDetail.getString("content"));
                } catch (Exception e) {
                    e.printStackTrace();
                }
                if (isNotExist(contentBiddingEntity.getCode(), contentBiddingEntity.getPubDate(), "ZJ")) {
                    dataQueue.add(contentBiddingEntity);
                }
                log.info("当前队列:{}", dataQueue.size());


            }
        } while (taskStatusMap.get("ZJ").getRun().get());
    }
    private synchronized JSONObject getNoticeDetail(String articleId) {
        articleId = articleId.replace("+", "%2B").replace("=", "%3D");
        sleep();
        try {
            String api = "https://zfcg.czt.zj.gov.cn/portal/detail?articleId=" + articleId + "&timestamp=" + new Date().getTime();
            log.info("get detail:" + api);
            String respStr = okHttpUtils.okhttpGet(api);
            JSONObject resp = JSONObject.parseObject(respStr).getJSONObject("result").getJSONObject("data");
            if (resp != null) {
                return resp;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new JSONObject();

    }

    protected void BJProduce() {
        String baseUrl = "http://ggzyfw.beijing.gov.cn";
        final AtomicInteger pageNumber = new AtomicInteger(1);
        String listUrl = baseUrl + "/jyxxrjxxzbgg/index.html";
        int errorCount = 0;
        do {
            // 招投标
            // 采购意向
            Element body = null;
            while (body == null) {
                try {
                    body = Jsoup.connect(listUrl).timeout(30000).get();
                } catch (Exception e) {
                    e.printStackTrace();
                    errorCount++;
                    if (errorCount == 30) {
                        return;
                    }
                }
            }

//            JSONArray array = JSON.parseObject(okHttpUtils.okhttpGet(api)).getJSONArray("article-listjy2");
            List<Element> titleList = body.select("#cmsContent > ul > li");

            if (titleList.size() == 0 || taskStatusMap.get("BJ").getRepeatCount().get() > 300) {
                taskStatusMap.get("BJ").getRun().set(false);
                return;
            }

            titleList.forEach(title -> {
                ContentBiddingEntity contentBiddingEntity = new ContentBiddingEntity();
                Element a = title.select("li > a").get(0);
                String uri = a.attr("href");
                String url = baseUrl + uri;
                String titleName = a.attr("title");
                String districtName = title.select("li > a > span > p").text();
                districtName = districtName.substring(districtName.indexOf("【") + 1, districtName.indexOf("】"));
                contentBiddingEntity.setDistrictName(districtName);
                contentBiddingEntity.setUrl(url);
                contentBiddingEntity.setTitle(titleName);
                Document contentHtml = null;
                try {
                    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:hh:ss");
                    contentHtml = Jsoup.connect(url).get();
                    contentBiddingEntity.setContentHtml(contentHtml.html());
                    contentBiddingEntity.setProvince("北京政府采购网");
                    contentBiddingEntity.setKeywords(contentHtml.select("head > meta:nth-child(13)").attr("content"));
                    contentBiddingEntity.setTypeName(contentHtml.select("head > meta:nth-child(11)").attr("content"));
                    contentBiddingEntity.setPubDate(simpleDateFormat.parse(contentHtml.select("head > meta:nth-child(16)").attr("content")).getTime());
                    contentBiddingEntity.setMainBidMenuName(contentHtml.select("head > meta:nth-child(12)").attr("content"));
                    contentBiddingEntity.setCode(contentHtml.select("body > div.content.clearfix > div.div-content > div.div-title > p").text().split("：")[1]);
                    contentBiddingEntity.setProjectCode(contentHtml.select("body > div.content.clearfix > div.div-content > div.div-article2 > div > div > div.ggxx-info > table > tbody > tr:nth-child(6) > td.s3a90a02d > div").attr("content"));
                    contentBiddingEntity.setProjectName(contentHtml.select("body > div.content.clearfix > div.div-content > div.div-article2 > div > div > div.ggxx-info > table > tbody > tr:nth-child(5) > td.s3a90a02d > div").attr("content"));
                    Thread.sleep(500);
                } catch (Exception e) {
                    e.printStackTrace();
                }

                if (isNotExist(contentBiddingEntity.getCode(), contentBiddingEntity.getPubDate(), "BJ")) {
                    dataQueue.add(contentBiddingEntity);
                }
                log.info("当前队列:{}", dataQueue.size());
            });
            listUrl = baseUrl + "/jyxxrjxxzbgg/index_" + pageNumber.incrementAndGet() + ".html";

        } while (taskStatusMap.get("BJ").getRun().get());
    }

    protected void JSProduce() throws InterruptedException {
        final AtomicInteger pageNumber = new AtomicInteger(1);

        String url = "https://www.jszbtb.com/#/bulletin?page=" + pageNumber + "&category_id=88&industry=G20&t=1654050940303&category_index=1&prev_categoryid=88";
        WebDriver driver = seleniumConfig.init();

        do {
            SeleniumUtil.to(driver, url);
            AtomicInteger errorCount = new AtomicInteger(0);

//            // 招投标
//            // 采购意向
//            Element body = null;
//            try {
//                body = Jsoup.connect(listUrl).timeout(30000).get();
//            } catch (Exception e) {
//                e.printStackTrace();
//                errorCount++;
//                if (errorCount==50) {
//                    return;
//                }
//                continue;
//            }
//            JSONArray array = JSON.parseObject(okHttpUtils.okhttpGet(api)).getJSONArray("article-listjy2");
            Thread.sleep(1000L);
            Elements trs = null;
            while ((trs == null || trs.size() == 0) && errorCount.get() < 20) {
                try {
                    String html = SeleniumUtil.findElement(driver, "/html").getAttribute("outerHTML");
                    Element body = Jsoup.parse(html).body();
                    trs = body.select("#app > div > div > div > div.bulletin-list-wrapper > div.bulletin-table.ivu-table-wrapper > div > div.ivu-table-body > table > tbody > tr");

                } catch (Exception e) {
                    errorCount.getAndIncrement();
                    e.printStackTrace();
                    Thread.sleep(1000L);
                }
            }

            if ((trs == null || trs.size() == 0) || taskStatusMap.get("JS").getRepeatCount().get() > 300) {
                taskStatusMap.get("JS").getRun().set(false);
                return;
            }
            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:hh:ss");

            trs.forEach(tr -> {
                ContentBiddingEntity contentBiddingEntity = new ContentBiddingEntity();
                Elements tds = tr.select("tr > td");
                contentBiddingEntity.setTitle(tds.get(0).select("td > div > div").text());
                contentBiddingEntity.setMainBidMenuName(tds.get(1).select("td > div > div").text());
                contentBiddingEntity.setDistrictName(tds.get(2).select("td > div > div").text());
                contentBiddingEntity.setProjectName(contentBiddingEntity.getTitle());
//                String uri = a.attr("href");
//                String url = baseUrl + uri;
//                String titleName = a.attr("title");
//                String districtName = title.select("li > a > span > p").text();
                SeleniumUtil.clickByCSS(driver, tds.get(0).select(".ivu-table-cell").get(0).cssSelector());
                Set<String> windowHandles = driver.getWindowHandles();
                WebDriver.TargetLocator targetLocator = driver.switchTo();
                String handle = "";
                for (String windowHandle : windowHandles) {
                    handle = windowHandle;
                }
                targetLocator.window(handle);


                try {
                    String code = null;
                    while (!StringUtils.hasLength(code) && errorCount.get() < 30) {
                        try {
//                            code = contentHtml.select("#app > div > section > div.bulletin-content-container > div.content-header > div > div:nth-child(1) > span.item-value").text();
                            code = SeleniumUtil.findElement(driver, "//*[@id=\"app\"]/div/section/div[2]/div[1]/div/div[1]/span[2]").getText();
                        } catch (Exception e) {
                            errorCount.getAndIncrement();
                            e.printStackTrace();
                            Thread.sleep(1000L);
                        }
                    }
                    long pubDate = simpleDateFormat.parse(code).getTime();
                    contentBiddingEntity.setCode(String.valueOf(pubDate));
                    contentBiddingEntity.setUrl(driver.getCurrentUrl());
                    SeleniumUtil.closeWindow(driver);
                    contentBiddingEntity.setPubDate(pubDate);
                    contentBiddingEntity.setProvince("江苏政府采购网");
                    contentBiddingEntity.setTypeName("招标公告");
//                    contentBiddingEntity.setProjectCode(contentHtml.select("body > div.content.clearfix > div.div-content > div.div-article2 > div > div > div.ggxx-info > table > tbody > tr:nth-child(6) > td.s3a90a02d > div").attr("content"));
//                    contentBiddingEntity.setProjectName(contentHtml.select("#app > div > section > div.bulletin-content-container > div.content-header > h1").text());
                    Thread.sleep(5000);

                } catch (Exception e) {
                    e.printStackTrace();
                }

                if (isNotExist(contentBiddingEntity.getCode(), contentBiddingEntity.getPubDate(), "JS")) {
                    dataQueue.add(contentBiddingEntity);
                }
                log.info("当前队列:{}", dataQueue.size());
            });
//            listUrl = baseUrl + "/jyxxrjxxzbgg/index_"+ pageNumber.incrementAndGet()  +".html";
            url = "https://www.jszbtb.com/#/bulletin?page=" + pageNumber.incrementAndGet() + "&category_id=88&industry=G20&t=1654050940303&category_index=1&prev_categoryid=88";

        } while (taskStatusMap.get("JS").getRun().get());
    }

    private boolean isNotExist(String code, Long pubDate, String name) {
        List<ContentBiddingEntity> contentBiddingEntityList = mapper.selectList(new QueryWrapper<ContentBiddingEntity>().eq("code", code).eq("pubDate", pubDate));
        if (!contentBiddingEntityList.isEmpty()) {
            taskStatusMap.get(name).getRepeatCount().incrementAndGet();
            log.info(name + "重复数据数量 " + taskStatusMap.get(name).getRepeatCount().get());
            return false;
        }
        return true;
    }
    private void sleep() {
        try {
            Thread.sleep((long) (Math.random() * 5000) + 30000);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
}


@Getter
class TaskStatus {
    private final AtomicInteger repeatCount = new AtomicInteger(0);
    private final AtomicBoolean run = new AtomicBoolean(false);
}
